from scrapy_redis.dupefilter import RFPDupeFilter
from scrapy import Request


class HouseDupeFilter(RFPDupeFilter):

    def request_seen(self, request):

        # https://zu.fang.com/hezu/3_431781862_1.htm
        if '_' in request.url and 'fang.com' in request.url:
            # 获取request的url中的房源编码
            url = request.url
            housenumber = url.split('_')[1]
            part_url = url.rsplit('/', 1)[0]
            url = part_url + '/' + housenumber
            # 获取request的请求体
            body_content = request.body
            # 获取request的请求方法
            method_type = request.method
            request = Request(url=url, body=body_content, method=method_type)
        print(request.url)
        fp = self.request_fingerprint(request)
        # This returns the number of values added, zero if already exists.
        added = self.server.sadd(self.key, fp)
        return added == 0


if __name__ == '__main__':
    r_url = 'https://zu.fang.com/hezu/3_431781862_1.htm'
    house_number = r_url.split('_')[1]
    print(house_number)
    housenumber = r_url.split('_')[1]
    part_url = r_url.rsplit('/', 1)[0]
    url = part_url + '/' + housenumber
    print(url)

    from w3lib.url import canonicalize_url

    result = canonicalize_url(house_number)

    print(result)

